{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/lamb_optimizer.py:34: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from albert import modeling\n",
    "from albert import optimization\n",
    "from albert import tokenization\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/tokenization.py:240: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n",
      "INFO:tensorflow:loading sentence piece model\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tokenization.FullTokenizer(\n",
    "      vocab_file='albert-base-2020-04-10/sp10m.cased.v10.vocab', do_lower_case=False,\n",
    "      spm_model_file='albert-base-2020-04-10/sp10m.cased.v10.model')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('train_X.pkl', 'rb') as fopen:\n",
    "    train_X, train_Y, train_depends = pickle.load(fopen)\n",
    "    \n",
    "with open('test_X.pkl', 'rb') as fopen:\n",
    "    test_X, test_Y, test_depends = pickle.load(fopen)\n",
    "    \n",
    "with open('tags.pkl', 'rb') as fopen:\n",
    "    idx2tag, tag2idx = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:116: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<albert.modeling.AlbertConfig at 0x7f2901d82d68>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "albert_config = modeling.AlbertConfig.from_json_file('albert-tiny-2020-04-17/config.json')\n",
    "albert_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'albert-tiny-2020-04-17/model.ckpt-1000000'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 3\n",
    "batch_size = 32\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(train_X) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiAAttention:\n",
    "    def __init__(self, input_size_encoder, input_size_decoder, num_labels):\n",
    "        self.input_size_encoder = input_size_encoder\n",
    "        self.input_size_decoder = input_size_decoder\n",
    "        self.num_labels = num_labels\n",
    "        \n",
    "        self.W_d = tf.get_variable(\"W_d\", shape=[self.num_labels, self.input_size_decoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_e = tf.get_variable(\"W_e\", shape=[self.num_labels, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.U = tf.get_variable(\"U\", shape=[self.num_labels, self.input_size_decoder, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        \n",
    "    def forward(self, input_d, input_e, mask_d=None, mask_e=None):\n",
    "        batch = tf.shape(input_d)[0]\n",
    "        length_decoder = tf.shape(input_d)[1]\n",
    "        length_encoder = tf.shape(input_e)[1]\n",
    "        out_d = tf.expand_dims(tf.matmul(self.W_d, tf.transpose(input_d, [0, 2, 1])), 3)\n",
    "        out_e = tf.expand_dims(tf.matmul(self.W_e, tf.transpose(input_e, [0, 2, 1])), 2)\n",
    "        output = tf.matmul(tf.expand_dims(input_d, 1), self.U)\n",
    "        output = tf.matmul(output, tf.transpose(tf.expand_dims(input_e, 1), [0, 1, 3, 2]))\n",
    "        \n",
    "        output = output + out_d + out_e\n",
    "        \n",
    "        if mask_d is not None:\n",
    "            d = tf.expand_dims(tf.expand_dims(mask_d, 1), 3)\n",
    "            e = tf.expand_dims(tf.expand_dims(mask_e, 1), 2)\n",
    "            output = output * d * e\n",
    "            \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiLinear:\n",
    "    def __init__(self, left_features, right_features, out_features):\n",
    "        self.left_features = left_features\n",
    "        self.right_features = right_features\n",
    "        self.out_features = out_features\n",
    "        \n",
    "        self.U = tf.get_variable(\"U-bi\", shape=[out_features, left_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_l = tf.get_variable(\"Wl\", shape=[out_features, left_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_r = tf.get_variable(\"Wr\", shape=[out_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    def forward(self, input_left, input_right):\n",
    "        left_size = tf.shape(input_left)\n",
    "        output_shape = tf.concat([left_size[:-1], [self.out_features]], axis = 0)\n",
    "        batch = tf.cast(tf.reduce_prod(left_size[:-1]), tf.int32)\n",
    "        input_left = tf.reshape(input_left, (batch, self.left_features))\n",
    "        input_right = tf.reshape(input_right, (batch, self.right_features))\n",
    "        tiled = tf.tile(tf.expand_dims(input_left, axis = 0), (self.out_features,1,1))\n",
    "        output = tf.transpose(tf.reduce_sum(tf.matmul(tiled, self.U), axis = 2))\n",
    "        output = output + tf.matmul(input_left, tf.transpose(self.W_l))\\\n",
    "        + tf.matmul(input_right, tf.transpose(self.W_r))\n",
    "        \n",
    "        return tf.reshape(output, output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "_NEG_INF = -1e9\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        learning_rate,\n",
    "        hidden_size_word,\n",
    "        training = True,\n",
    "        cov = 0.0):\n",
    "        \n",
    "        self.words = tf.placeholder(tf.int32, (None, None))\n",
    "        self.heads = tf.placeholder(tf.int32, (None, None))\n",
    "        self.types = tf.placeholder(tf.int32, (None, None))\n",
    "        self.switch = tf.placeholder(tf.bool, None)\n",
    "        self.mask = tf.cast(tf.math.not_equal(self.words, 0), tf.float32)\n",
    "        self.maxlen = tf.shape(self.words)[1]\n",
    "        self.lengths = tf.count_nonzero(self.words, 1)\n",
    "        mask = self.mask\n",
    "        heads = self.heads\n",
    "        types = self.types\n",
    "        \n",
    "        self.arc_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.arc_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.attention = BiAAttention(hidden_size_word, hidden_size_word, 1)\n",
    "\n",
    "        self.type_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.type_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.bilinear = BiLinear(hidden_size_word, hidden_size_word, len(tag2idx))\n",
    "        \n",
    "        model = modeling.AlbertModel(\n",
    "            config=albert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.words,\n",
    "            input_mask=self.mask,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        output_layer = model.get_sequence_output()\n",
    "        \n",
    "        arc_h = tf.nn.elu(self.arc_h(output_layer))\n",
    "        arc_c = tf.nn.elu(self.arc_c(output_layer))\n",
    "        self._arc_h = arc_h\n",
    "        self._arc_c = arc_c\n",
    "        \n",
    "        type_h = tf.nn.elu(self.type_h(output_layer))\n",
    "        type_c = tf.nn.elu(self.type_c(output_layer))\n",
    "        self._type_h = type_h\n",
    "        self._type_c = type_c\n",
    "        \n",
    "        out_arc = tf.squeeze(self.attention.forward(arc_h, arc_c, mask_d=self.mask, \n",
    "                                                    mask_e=self.mask), axis = 1)\n",
    "        self.out_arc = out_arc\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        \n",
    "        decode_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        decode_arc = tf.where(minus_mask, tf.fill(tf.shape(decode_arc), -np.inf), decode_arc)\n",
    "        self.decode_arc = decode_arc\n",
    "        self.heads_seq = tf.argmax(decode_arc, axis = 1)\n",
    "        self.heads_seq = tf.identity(self.heads_seq, name = 'heads_seq')\n",
    "        \n",
    "#         self.decode_arc_t = tf.transpose(decode_arc, (0, 2, 1))\n",
    "#         sequence_loss_depends = tf.contrib.seq2seq.sequence_loss(logits = self.decode_arc_t,\n",
    "#                                                      targets = self.heads,\n",
    "#                                                      weights = mask)\n",
    "        \n",
    "        t = tf.cast(tf.transpose(self.heads_seq), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        self.tags_seq = tf.argmax(out_type, axis = 2)\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'tags_seq')\n",
    "        \n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            out_type, self.types, self.lengths\n",
    "        )\n",
    "        crf_loss = tf.reduce_mean(-log_likelihood)\n",
    "        self.logits, _ = tf.contrib.crf.crf_decode(\n",
    "            out_type, transition_params, self.lengths\n",
    "        )\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        minus_inf = -1e8\n",
    "        minus_mask = (1 - mask) * minus_inf\n",
    "        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n",
    "        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n",
    "        loss_type = tf.nn.log_softmax(out_type, dim=2)\n",
    "        loss_arc = loss_arc * tf.expand_dims(mask, axis = 2) * tf.expand_dims(mask, axis = 1)\n",
    "        loss_type = loss_type * tf.expand_dims(mask, axis = 2)\n",
    "        num = tf.reduce_sum(mask) - tf.cast(batch, tf.float32)\n",
    "        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n",
    "        loss_arc = tf.gather_nd(loss_arc, concatenated)\n",
    "        loss_arc = tf.transpose(loss_arc, [1, 0])\n",
    "        \n",
    "        t = tf.transpose(types)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        loss_type = tf.gather_nd(loss_type, concatenated)\n",
    "        loss_type = tf.transpose(loss_type, [1, 0])\n",
    "        cost = (tf.reduce_sum(-loss_arc) / num) + (tf.reduce_sum(-loss_type) / num)\n",
    "        \n",
    "        self.cost = tf.cond(self.switch, lambda: cost + crf_loss, lambda: cost)\n",
    "        self.optimizer = optimization.create_optimizer(self.cost, learning_rate, \n",
    "                                                       num_train_steps, num_warmup_steps, False)\n",
    "        \n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        \n",
    "        self.prediction = tf.boolean_mask(self.logits, mask)\n",
    "        mask_label = tf.boolean_mask(self.types, mask)\n",
    "        correct_pred = tf.equal(tf.cast(self.prediction, tf.int32), mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        self.prediction = tf.cast(tf.boolean_mask(self.heads_seq, mask), tf.int32)\n",
    "        mask_label = tf.boolean_mask(self.heads, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:194: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:588: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:1025: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/modeling.py:253: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From <ipython-input-10-4486bc1f4ea9>:61: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From <ipython-input-10-4486bc1f4ea9>:101: calling log_softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/optimization.py:36: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/optimization.py:41: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "INFO:tensorflow:++++++ warmup starts at step 0, for 29336 steps ++++++\n",
      "INFO:tensorflow:using adamw\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/albert/optimization.py:101: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from albert-tiny-2020-04-17/model.ckpt-1000000\n"
     ]
    }
   ],
   "source": [
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "batch_x = train_X[:5]\n",
    "batch_x = pad_sequences(batch_x,padding='post')\n",
    "batch_y = train_Y[:5]\n",
    "batch_y = pad_sequences(batch_y,padding='post')\n",
    "batch_depends = train_depends[:5]\n",
    "batch_depends = pad_sequences(batch_depends,padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0, 0.09375, 29.16653]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.switch: False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0, 0.09375, 120.98015]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.switch: True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:  66%|██████▌   | 64243/97788 [2:44:46<1:39:56,  5.59it/s, accuracy=0.867, accuracy_depends=0.726, cost=0.871] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  85%|████████▌ | 83568/97788 [3:34:59<34:25,  6.88it/s, accuracy=0.87, accuracy_depends=0.753, cost=0.777]   IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop: 100%|██████████| 97788/97788 [4:12:02<00:00,  6.47it/s, accuracy=0.887, accuracy_depends=0.825, cost=0.534]  \n",
      "test minibatch loop: 100%|██████████| 313/313 [00:33<00:00,  9.35it/s, accuracy=0.881, accuracy_depends=0.833, cost=0.496]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 1.188455, training acc: 0.822894, training depends: 0.697532, valid loss: 0.690450, valid acc: 0.879138, valid depends: 0.771131\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:  29%|██▉       | 28730/97788 [1:20:49<2:43:21,  7.05it/s, accuracy=0.878, accuracy_depends=0.798, cost=0.597]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  58%|█████▊    | 56970/97788 [2:40:22<1:41:24,  6.71it/s, accuracy=0.89, accuracy_depends=0.816, cost=0.497]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  62%|██████▏   | 60199/97788 [2:49:11<1:24:55,  7.38it/s, accuracy=0.878, accuracy_depends=0.786, cost=0.672]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  65%|██████▌   | 63584/97788 [2:58:16<1:39:38,  5.72it/s, accuracy=0.868, accuracy_depends=0.822, cost=0.511]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  68%|██████▊   | 66826/97788 [3:06:56<1:39:53,  5.17it/s, accuracy=0.873, accuracy_depends=0.804, cost=0.545]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  72%|███████▏  | 70225/97788 [3:15:54<1:01:21,  7.49it/s, accuracy=0.881, accuracy_depends=0.768, cost=0.641]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  75%|███████▌  | 73415/97788 [3:24:27<55:26,  7.33it/s, accuracy=0.907, accuracy_depends=0.738, cost=0.964]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  79%|███████▊  | 76845/97788 [3:33:36<1:00:03,  5.81it/s, accuracy=0.889, accuracy_depends=0.799, cost=0.525]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  82%|████████▏ | 80161/97788 [3:43:31<44:58,  6.53it/s, accuracy=0.863, accuracy_depends=0.777, cost=0.801]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop: 100%|██████████| 97788/97788 [4:30:08<00:00,  6.03it/s, accuracy=0.895, accuracy_depends=0.831, cost=0.433]  \n",
      "test minibatch loop: 100%|██████████| 313/313 [00:35<00:00,  8.72it/s, accuracy=0.892, accuracy_depends=0.857, cost=0.382]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, training loss: 0.631874, training acc: 0.882788, training depends: 0.782693, valid loss: 0.586480, valid acc: 0.886249, valid depends: 0.789142\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 2\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'albert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 97788/97788 [4:21:31<00:00,  6.23it/s, accuracy=0.934, accuracy_depends=0.823, cost=3.76]   \n",
      "test minibatch loop: 100%|██████████| 313/313 [00:36<00:00,  8.57it/s, accuracy=0.958, accuracy_depends=0.844, cost=3.73]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 5.254430, training acc: 0.932902, training depends: 0.787293, valid loss: 4.992470, valid acc: 0.935338, valid depends: 0.786505\n",
      "\n"
     ]
    }
   ],
   "source": [
    "epoch = 1\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'albert-tiny-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_sentencepiece_tokens_tagging(x, y):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(x)\n",
    "    rejected = ['[CLS]', '[SEP]']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_label = x[i], y[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_label = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_label = [previous_label]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_label.append(current_label)\n",
    "                i = i + 1\n",
    "                current_token, current_label = x[i], y[i]\n",
    "            merged_label = merged_label[0]\n",
    "            new_paired_tokens.append((merged_token, merged_label))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_label))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in rejected\n",
    "    ]\n",
    "    labels = [i[1] for i in new_paired_tokens if i[0] not in rejected]\n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from malaya.function.parse_dependency import DependencyGraph\n",
    "\n",
    "PUNCTUATION = '!\"#$%&\\'()*+,./:;<=>?@[\\]^_`{|}~'\n",
    "\n",
    "def transformer_textcleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    string = re.sub('\\\\(dot\\\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    string = ' '.join(string)\n",
    "    string = re.sub(f'([{PUNCTUATION}])', r' \\1 ', string)\n",
    "    string = re.sub('\\s{2,}', ' ', string)\n",
    "    original_string = string.split()\n",
    "    string = [\n",
    "        (original_string[no], word.title() if word.isupper() else word)\n",
    "        for no, word in enumerate(string.split())\n",
    "        if len(word)\n",
    "    ]\n",
    "    return [s[0] for s in string], [s[1] for s in string]\n",
    "\n",
    "def parse_X(left):\n",
    "    bert_tokens = ['[CLS]']\n",
    "    for no, orig_token in enumerate(left):\n",
    "        t = tokenizer.tokenize(orig_token)\n",
    "        bert_tokens.extend(t)\n",
    "    bert_tokens.append(\"[SEP]\")\n",
    "    t = tokenizer.convert_tokens_to_ids(bert_tokens)\n",
    "    return t, bert_tokens, [1] * len(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dependency_graph(tagging, indexing):\n",
    "    \"\"\"\n",
    "    Return helper object for dependency parser results. Only accept tagging and indexing outputs from dependency models.\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    for i in range(len(tagging)):\n",
    "        result.append(\n",
    "            '%d\\t%s\\t_\\t_\\t_\\t_\\t%d\\t%s\\t_\\t_'\n",
    "            % (i + 1, tagging[i][0], int(indexing[i][1]), tagging[i][1])\n",
    "        )\n",
    "    return DependencyGraph('\\n'.join(result), top_relation_label='root')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"193pt\" height=\"218pt\"\n",
       " viewBox=\"0.00 0.00 192.50 218.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 214)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-214 188.5,-214 188.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"94\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"94\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (makan)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M94,-173.9735C94,-162.1918 94,-146.5607 94,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"97.5001,-133.0033 94,-123.0034 90.5001,-133.0034 97.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"109\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (husein)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;1 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>2&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M83.64,-86.9735C76.6677,-74.8418 67.3498,-58.6287 59.4999,-44.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"62.3642,-42.9295 54.3468,-36.0034 56.2951,-46.4175 62.3642,-42.9295\"/>\n",
       "<text text-anchor=\"middle\" x=\"92\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (ayam)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M105.3041,-86.7532C108.7732,-81.0866 112.5772,-74.8047 116,-69 120.611,-61.1804 125.5263,-52.6241 129.9752,-44.7918\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"133.0613,-46.4448 134.9374,-36.0174 126.9682,-42.9989 133.0613,-46.4448\"/>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f29e298e208>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'husein makan ayam'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, mask = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(bert_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(bert_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"2514pt\" height=\"827pt\"\n",
       " viewBox=\"0.00 0.00 2513.50 827.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 823)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-823 2509.5,-823 2509.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"465\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"299\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (Kuala)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M430.6047,-782.9735C405.0085,-769.5587 369.8902,-751.1533 342.3687,-736.7294\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"343.8333,-733.5454 333.3512,-732.0034 340.5838,-739.7455 343.8333,-733.5454\"/>\n",
       "<text text-anchor=\"middle\" x=\"407\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 13 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>13</title>\n",
       "<text text-anchor=\"middle\" x=\"630\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">13 (membidas)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;13 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;13</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M499.1881,-782.9735C524.6301,-769.5587 559.5368,-751.1533 586.8926,-736.7294\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"588.6424,-739.7635 595.8557,-732.0034 585.3775,-733.5715 588.6424,-739.7635\"/>\n",
       "<text text-anchor=\"middle\" x=\"573\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"85\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (Lumpur)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M258.4967,-697.5337C224.6817,-683.7865 176.1868,-664.0713 139.06,-648.9777\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"139.9796,-645.5734 129.3978,-645.0496 137.3433,-652.058 139.9796,-645.5734\"/>\n",
       "<text text-anchor=\"middle\" x=\"217\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<text text-anchor=\"middle\" x=\"251\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M281.7697,-695.8781C277.1896,-690.4224 272.5388,-684.2237 269,-678 264.934,-670.8491 261.5428,-662.6197 258.8408,-654.8873\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"262.0959,-653.578 255.687,-645.1414 255.4359,-655.7332 262.0959,-653.578\"/>\n",
       "<text text-anchor=\"middle\" x=\"290.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 37 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>37</title>\n",
       "<text text-anchor=\"middle\" x=\"337\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">37 (.)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;37 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>1&#45;&gt;37</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M306.8736,-695.9735C312.1216,-683.9585 319.1181,-667.9401 325.0477,-654.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"328.3411,-655.5683 329.1365,-645.0034 321.9263,-652.7664 328.3411,-655.5683\"/>\n",
       "<text text-anchor=\"middle\" x=\"341.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 14 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>14</title>\n",
       "<text text-anchor=\"middle\" x=\"523\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">14 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;14 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>13&#45;&gt;14</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M607.8295,-695.9735C592.0481,-683.1419 570.6511,-665.7443 553.2953,-651.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"555.1091,-648.5964 545.1421,-645.0034 550.693,-654.0277 555.1091,-648.5964\"/>\n",
       "<text text-anchor=\"middle\" x=\"594\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 23 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>23</title>\n",
       "<text text-anchor=\"middle\" x=\"630\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">23 (,)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;23 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>13&#45;&gt;23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M630,-695.9735C630,-684.1918 630,-668.5607 630,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"633.5001,-655.0033 630,-645.0034 626.5001,-655.0034 633.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"650.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 39 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>39</title>\n",
       "<text text-anchor=\"middle\" x=\"727\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">39 (berkata)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;39 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>13&#45;&gt;39</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M653.7251,-695.7538C660.6992,-690.1952 668.2615,-683.9726 675,-678 683.9839,-670.0372 693.456,-660.9664 701.8186,-652.7045\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"704.5334,-654.9401 709.1409,-645.3995 699.5895,-649.9845 704.5334,-654.9401\"/>\n",
       "<text text-anchor=\"middle\" x=\"705.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 46 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>46</title>\n",
       "<text text-anchor=\"middle\" x=\"1101\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">46 (memetik)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;46 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>13&#45;&gt;46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M691.575,-702.6263C780.3828,-686.2223 943.7466,-656.0468 1034.9727,-639.1961\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1035.7225,-642.6169 1044.9204,-637.3587 1034.4509,-635.7334 1035.7225,-642.6169\"/>\n",
       "<text text-anchor=\"middle\" x=\"907.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (:)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M72.9824,-608.9735C64.8168,-596.7252 53.8777,-580.3165 44.7177,-566.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"47.4615,-564.3824 39.0022,-558.0034 41.6371,-568.2653 47.4615,-564.3824\"/>\n",
       "<text text-anchor=\"middle\" x=\"80.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>5</title>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">5 (Penerangan)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M226.9647,-608.9735C209.6226,-595.9669 186.0254,-578.269 167.0762,-564.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"169.1045,-561.2034 159.0045,-558.0034 164.9045,-566.8034 169.1045,-561.2034\"/>\n",
       "<text text-anchor=\"middle\" x=\"212\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 9 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>9</title>\n",
       "<text text-anchor=\"middle\" x=\"251\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">9 (Wan)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;9 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>4&#45;&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M251,-608.9735C251,-597.1918 251,-581.5607 251,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"254.5001,-568.0033 251,-558.0034 247.5001,-568.0034 254.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>7</title>\n",
       "<text text-anchor=\"middle\" x=\"332\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">7 (,)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;7 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>4&#45;&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M267.7833,-608.9735C279.5127,-596.3752 295.3401,-579.3755 308.3466,-565.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"310.9857,-567.7073 315.2382,-558.0034 305.8624,-562.9374 310.9857,-567.7073\"/>\n",
       "<text text-anchor=\"middle\" x=\"317.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>6</title>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">6 (Bersatu)</text>\n",
       "</g>\n",
       "<!-- 5&#45;&gt;6 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>5&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M135,-521.9735C135,-510.1918 135,-494.5607 135,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"138.5001,-481.0033 135,-471.0034 131.5001,-481.0034 138.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"147\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 10 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>10</title>\n",
       "<text text-anchor=\"middle\" x=\"249\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">10 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 9&#45;&gt;10 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>9&#45;&gt;10</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M250.5856,-521.9735C250.3148,-510.1918 249.9554,-494.5607 249.6473,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"253.1429,-480.9202 249.4139,-471.0034 246.1447,-481.0812 253.1429,-480.9202\"/>\n",
       "<text text-anchor=\"middle\" x=\"263\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>8</title>\n",
       "<text text-anchor=\"middle\" x=\"162\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 11 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>11</title>\n",
       "<text text-anchor=\"middle\" x=\"249\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">11 (Wan)</text>\n",
       "</g>\n",
       "<!-- 10&#45;&gt;11 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>10&#45;&gt;11</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M249,-434.9735C249,-423.1918 249,-407.5607 249,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"252.5001,-394.0033 249,-384.0034 245.5001,-394.0034 252.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"261\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 12 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>12</title>\n",
       "<text text-anchor=\"middle\" x=\"249\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">12 (Jan)</text>\n",
       "</g>\n",
       "<!-- 11&#45;&gt;12 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>11&#45;&gt;12</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M249,-347.9735C249,-336.1918 249,-320.5607 249,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"252.5001,-307.0033 249,-297.0034 245.5001,-307.0034 252.5001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"261\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 12&#45;&gt;8 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>12&#45;&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M222.6648,-260.6799C215.6529,-255.3163 208.2861,-249.2144 202,-243 194.5166,-235.6019 187.1629,-226.8033 180.8427,-218.6146\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"183.4698,-216.2842 174.6684,-210.3862 177.8707,-220.4855 183.4698,-216.2842\"/>\n",
       "<text text-anchor=\"middle\" x=\"223.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 61 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>61</title>\n",
       "<text text-anchor=\"middle\" x=\"249\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">61 (.)</text>\n",
       "</g>\n",
       "<!-- 12&#45;&gt;61 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>12&#45;&gt;61</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M249,-260.9735C249,-249.1918 249,-233.5607 249,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"252.5001,-220.0033 249,-210.0034 245.5001,-220.0034 252.5001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"269.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 64 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>64</title>\n",
       "<text text-anchor=\"middle\" x=\"346\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">64 (berkata)</text>\n",
       "</g>\n",
       "<!-- 12&#45;&gt;64 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>12&#45;&gt;64</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M272.7251,-260.7538C279.6992,-255.1952 287.2615,-248.9726 294,-243 302.9839,-235.0372 312.456,-225.9664 320.8186,-217.7045\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"323.5334,-219.9401 328.1409,-210.3995 318.5895,-214.9845 323.5334,-219.9401\"/>\n",
       "<text text-anchor=\"middle\" x=\"324.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 15 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>15</title>\n",
       "<text text-anchor=\"middle\" x=\"436\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">15 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 14&#45;&gt;15 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>14&#45;&gt;15</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M504.9735,-608.9735C492.3752,-596.3752 475.3755,-579.3755 461.4055,-565.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"463.5493,-562.5996 454.0034,-558.0034 458.5996,-567.5493 463.5493,-562.5996\"/>\n",
       "<text text-anchor=\"middle\" x=\"501\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 20 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>20</title>\n",
       "<text text-anchor=\"middle\" x=\"558\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">20 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 14&#45;&gt;20 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>14&#45;&gt;20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M530.252,-608.9735C535.0857,-596.9585 541.5298,-580.9401 546.9913,-567.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"550.272,-568.5871 550.7573,-558.0034 543.7778,-565.9744 550.272,-568.5871\"/>\n",
       "<text text-anchor=\"middle\" x=\"558\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 38 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>38</title>\n",
       "<text text-anchor=\"middle\" x=\"715\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">38 (Beliau)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;38 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>39&#45;&gt;38</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M724.5136,-608.9735C722.8885,-597.1918 720.7325,-581.5607 718.8839,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"722.3169,-567.4313 717.4832,-558.0034 715.3825,-568.3878 722.3169,-567.4313\"/>\n",
       "<text text-anchor=\"middle\" x=\"742\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 40 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>40</title>\n",
       "<text text-anchor=\"middle\" x=\"808\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">40 (,)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;40 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>39&#45;&gt;40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M747.7433,-608.8228C753.7926,-603.2662 760.3039,-597.0273 766,-591 773.3268,-583.2471 780.8731,-574.3965 787.5125,-566.269\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"790.2628,-568.434 793.8079,-558.4499 784.8103,-564.0441 790.2628,-568.434\"/>\n",
       "<text text-anchor=\"middle\" x=\"800.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 43 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>43</title>\n",
       "<text text-anchor=\"middle\" x=\"983\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">43 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;43 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>46&#45;&gt;43</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1076.5503,-608.9735C1058.9092,-595.9669 1034.9051,-578.269 1015.6292,-564.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1017.5442,-561.1206 1007.4184,-558.0034 1013.3902,-566.7548 1017.5442,-561.1206\"/>\n",
       "<text text-anchor=\"middle\" x=\"1065\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 45 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>45</title>\n",
       "<text text-anchor=\"middle\" x=\"1101\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">45 (sengaja)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;45 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>46&#45;&gt;45</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1101,-608.9735C1101,-597.1918 1101,-581.5607 1101,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1104.5001,-568.0033 1101,-558.0034 1097.5001,-568.0034 1104.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1129.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 47 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>47</title>\n",
       "<text text-anchor=\"middle\" x=\"1337\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">47 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;47 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>46&#45;&gt;47</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1149.8994,-608.9735C1187.3176,-595.1795 1239.0491,-576.109 1278.6342,-561.5162\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1279.9912,-564.7463 1288.1633,-558.0034 1277.5699,-558.1784 1279.9912,-564.7463\"/>\n",
       "<text text-anchor=\"middle\" x=\"1245\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 51 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>51</title>\n",
       "<text text-anchor=\"middle\" x=\"1561\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">51 (Tan)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;51 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>46&#45;&gt;51</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1157.405,-616.3321C1248.9283,-599.0223 1428.1298,-565.1298 1513.5396,-548.9762\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1514.3029,-552.394 1523.4782,-547.0965 1513.002,-545.5159 1514.3029,-552.394\"/>\n",
       "<text text-anchor=\"middle\" x=\"1380.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 16 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>16</title>\n",
       "<text text-anchor=\"middle\" x=\"352\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16 (Seri)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;16 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>15&#45;&gt;16</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M418.5951,-521.9735C406.4312,-509.3752 390.0177,-492.3755 376.5295,-478.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"378.8464,-475.7663 369.3826,-471.0034 373.8106,-480.6285 378.8464,-475.7663\"/>\n",
       "<text text-anchor=\"middle\" x=\"412\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 19 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>19</title>\n",
       "<text text-anchor=\"middle\" x=\"448\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">19 (dan)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;19 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>20&#45;&gt;19</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M535.2079,-521.9735C518.8365,-509.0253 496.5867,-491.4276 478.6591,-477.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"480.7774,-474.4616 470.7629,-471.0034 476.435,-479.952 480.7774,-474.4616\"/>\n",
       "<text text-anchor=\"middle\" x=\"518\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 21 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>21</title>\n",
       "<text text-anchor=\"middle\" x=\"558\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">21 (Pemuda)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;21 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>20&#45;&gt;21</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M558,-521.9735C558,-510.1918 558,-494.5607 558,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"561.5001,-481.0033 558,-471.0034 554.5001,-481.0034 561.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"570\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 22 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>22</title>\n",
       "<text text-anchor=\"middle\" x=\"676\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">22 (Umno)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;22 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>20&#45;&gt;22</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M582.4497,-521.9735C600.0908,-508.9669 624.0949,-491.269 643.3708,-477.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"645.6098,-479.7548 651.5816,-471.0034 641.4558,-474.1206 645.6098,-479.7548\"/>\n",
       "<text text-anchor=\"middle\" x=\"637\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 24 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>24</title>\n",
       "<text text-anchor=\"middle\" x=\"786\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">24 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;24 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>20&#45;&gt;24</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M603.918,-522.6412C619.5029,-516.7384 637.0137,-510.094 653,-504 678.3868,-494.3225 706.479,-483.5585 730.2788,-474.4228\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"731.7724,-477.5985 739.8534,-470.7466 729.2633,-471.0636 731.7724,-477.5985\"/>\n",
       "<text text-anchor=\"middle\" x=\"711.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 17 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>17</title>\n",
       "<text text-anchor=\"middle\" x=\"351\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">17 (Najib)</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;17 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>16&#45;&gt;17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M351.7928,-434.9735C351.6574,-423.1918 351.4777,-407.5607 351.3237,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"354.8217,-393.9624 351.2069,-384.0034 347.8222,-394.043 354.8217,-393.9624\"/>\n",
       "<text text-anchor=\"middle\" x=\"364\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 18 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>18</title>\n",
       "<text text-anchor=\"middle\" x=\"459\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">18 (Razak)</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;18 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>16&#45;&gt;18</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M374.1705,-434.9735C389.9519,-422.1419 411.3489,-404.7443 428.7047,-390.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"431.307,-393.0277 436.8579,-384.0034 426.8909,-387.5964 431.307,-393.0277\"/>\n",
       "<text text-anchor=\"middle\" x=\"424\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 25 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>25</title>\n",
       "<text text-anchor=\"middle\" x=\"703\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">25 (Dr)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;25 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>24&#45;&gt;25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M768.8023,-434.9735C756.7833,-422.3752 740.5651,-405.3755 727.2375,-391.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"729.6108,-388.8228 720.1756,-384.0034 724.546,-393.6548 729.6108,-388.8228\"/>\n",
       "<text text-anchor=\"middle\" x=\"762\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 30 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>30</title>\n",
       "<text text-anchor=\"middle\" x=\"858\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">30 (mempertikaikan)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;30 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>24&#45;&gt;30</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M800.9184,-434.9735C811.2481,-422.4919 825.1532,-405.6899 836.6529,-391.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"839.4213,-393.9388 843.1007,-384.0034 834.0286,-389.4758 839.4213,-393.9388\"/>\n",
       "<text text-anchor=\"middle\" x=\"837.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 26 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>26</title>\n",
       "<text text-anchor=\"middle\" x=\"664\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">26 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 25&#45;&gt;26 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>25&#45;&gt;26</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M694.9192,-347.9735C689.5331,-335.9585 682.3525,-319.9401 676.2669,-306.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"679.3549,-304.6967 672.0705,-297.0034 672.9673,-307.5602 679.3549,-304.6967\"/>\n",
       "<text text-anchor=\"middle\" x=\"698\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 29 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>29</title>\n",
       "<text text-anchor=\"middle\" x=\"772\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">29 (yang)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;29 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>30&#45;&gt;29</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M840.1807,-347.9735C827.7272,-335.3752 810.9229,-318.3755 797.1135,-304.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"799.3157,-301.6547 789.7964,-297.0034 794.3374,-306.5757 799.3157,-301.6547\"/>\n",
       "<text text-anchor=\"middle\" x=\"841\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 31 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>31</title>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">31 (tindakan)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;31 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>30&#45;&gt;31</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M864.216,-347.9735C868.3189,-336.0751 873.7756,-320.2508 878.4274,-306.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"881.8408,-307.5981 881.7919,-297.0034 875.2232,-305.3161 881.8408,-307.5981\"/>\n",
       "<text text-anchor=\"middle\" x=\"886\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 33 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>33</title>\n",
       "<text text-anchor=\"middle\" x=\"1038\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">33 (melaksanakan)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;33 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>30&#45;&gt;33</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M895.2961,-347.9735C923.1717,-334.5003 961.4624,-315.9932 991.3625,-301.5415\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"993.2713,-304.5063 1000.7517,-297.0034 990.2251,-298.2039 993.2713,-304.5063\"/>\n",
       "<text text-anchor=\"middle\" x=\"982.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 27 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>27</title>\n",
       "<text text-anchor=\"middle\" x=\"664\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">27 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 26&#45;&gt;27 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>26&#45;&gt;27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M664,-260.9735C664,-249.1918 664,-233.5607 664,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"667.5001,-220.0033 664,-210.0034 660.5001,-220.0034 667.5001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"676\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 28 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>28</title>\n",
       "<text text-anchor=\"middle\" x=\"664\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">28 (Dusuki)</text>\n",
       "</g>\n",
       "<!-- 27&#45;&gt;28 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>27&#45;&gt;28</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M664,-173.9735C664,-162.1918 664,-146.5607 664,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"667.5001,-133.0033 664,-123.0034 660.5001,-133.0034 667.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"676\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 32 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>32</title>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">32 (kerajaan)</text>\n",
       "</g>\n",
       "<!-- 31&#45;&gt;32 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>31&#45;&gt;32</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M888,-260.9735C888,-249.1918 888,-233.5607 888,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"891.5001,-220.0033 888,-210.0034 884.5001,-220.0034 891.5001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"899\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 34 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>34</title>\n",
       "<text text-anchor=\"middle\" x=\"1038\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">34 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;34 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>33&#45;&gt;34</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1038,-260.9735C1038,-249.1918 1038,-233.5607 1038,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1041.5001,-220.0033 1038,-210.0034 1034.5001,-220.0034 1041.5001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1049\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 35 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>35</title>\n",
       "<text text-anchor=\"middle\" x=\"1038\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">35 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 34&#45;&gt;35 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>34&#45;&gt;35</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1038,-173.9735C1038,-162.1918 1038,-146.5607 1038,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1041.5001,-133.0033 1038,-123.0034 1034.5001,-133.0034 1041.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1075\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 36 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>36</title>\n",
       "<text text-anchor=\"middle\" x=\"1038\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">36 (penuh)</text>\n",
       "</g>\n",
       "<!-- 35&#45;&gt;36 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>35&#45;&gt;36</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1038,-86.9735C1038,-75.1918 1038,-59.5607 1038,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1041.5001,-46.0033 1038,-36.0034 1034.5001,-46.0034 1041.5001,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1058\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 41 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>41</title>\n",
       "<text text-anchor=\"middle\" x=\"1095\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">41 (Najib)</text>\n",
       "</g>\n",
       "<!-- 42 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>42</title>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">42 (dan)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;42 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>43&#45;&gt;42</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M963.3159,-521.9735C949.4318,-509.2586 930.652,-492.0603 915.3188,-478.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"917.3974,-475.1759 907.6588,-471.0034 912.6698,-480.3383 917.3974,-475.1759\"/>\n",
       "<text text-anchor=\"middle\" x=\"950\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 44 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>44</title>\n",
       "<text text-anchor=\"middle\" x=\"989\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">44 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;44 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>43&#45;&gt;44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M984.2432,-521.9735C985.0557,-510.1918 986.1337,-494.5607 987.0581,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"990.562,-481.2205 987.7584,-471.0034 983.5786,-480.7388 990.562,-481.2205\"/>\n",
       "<text text-anchor=\"middle\" x=\"999\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 45&#45;&gt;41 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>45&#45;&gt;41</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1099.7568,-521.9735C1098.9443,-510.1918 1097.8663,-494.5607 1096.9419,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1100.4214,-480.7388 1096.2416,-471.0034 1093.438,-481.2205 1100.4214,-480.7388\"/>\n",
       "<text text-anchor=\"middle\" x=\"1119\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 48 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>48</title>\n",
       "<text text-anchor=\"middle\" x=\"1211\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">48 (Perdana)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;48 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>47&#45;&gt;48</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1310.8927,-521.9735C1291.9711,-508.9086 1266.1941,-491.1102 1245.5648,-476.8662\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1247.2915,-473.8052 1237.0738,-471.0034 1243.3141,-479.5655 1247.2915,-473.8052\"/>\n",
       "<text text-anchor=\"middle\" x=\"1294\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 49 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>49</title>\n",
       "<text text-anchor=\"middle\" x=\"1337\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">49 (Menteri)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;49 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>47&#45;&gt;49</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1337,-521.9735C1337,-510.1918 1337,-494.5607 1337,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1340.5001,-481.0033 1337,-471.0034 1333.5001,-481.0034 1340.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1349\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 50 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>50</title>\n",
       "<text text-anchor=\"middle\" x=\"1436\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">50 (,)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;50 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>47&#45;&gt;50</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1357.5129,-521.9735C1372.1144,-509.1419 1391.9116,-491.7443 1407.9698,-477.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1410.3122,-480.2336 1415.5134,-471.0034 1405.6913,-474.9755 1410.3122,-480.2336\"/>\n",
       "<text text-anchor=\"middle\" x=\"1413.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 57 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>57</title>\n",
       "<text text-anchor=\"middle\" x=\"1534\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">57 (lengkap)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;57 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>47&#45;&gt;57</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1392.3481,-521.9112C1407.3359,-516.5706 1423.4373,-510.4225 1438,-504 1456.6492,-495.7752 1476.6491,-485.4132 1493.5409,-476.1811\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1495.5164,-479.0883 1502.5771,-471.1891 1492.1315,-472.9611 1495.5164,-479.0883\"/>\n",
       "<text text-anchor=\"middle\" x=\"1488\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 53 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>53</title>\n",
       "<text text-anchor=\"middle\" x=\"1668\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">53 (Muhyiddin)</text>\n",
       "</g>\n",
       "<!-- 51&#45;&gt;53 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>51&#45;&gt;53</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1583.1705,-521.9735C1598.9519,-509.1419 1620.3489,-491.7443 1637.7047,-477.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1640.307,-480.0277 1645.8579,-471.0034 1635.8909,-474.5964 1640.307,-480.0277\"/>\n",
       "<text text-anchor=\"middle\" x=\"1633\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 55 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>55</title>\n",
       "<text text-anchor=\"middle\" x=\"1431\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">55 (yang)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;55 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>57&#45;&gt;55</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1511.5195,-434.8023C1504.5864,-429.1378 1496.9493,-422.8446 1490,-417 1479.9619,-408.5576 1469.0721,-399.2015 1459.4128,-390.8322\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1461.479,-387.9909 1451.634,-384.0758 1456.8887,-393.2758 1461.479,-387.9909\"/>\n",
       "<text text-anchor=\"middle\" x=\"1510\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 56 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>56</title>\n",
       "<text text-anchor=\"middle\" x=\"1534\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">56 (tidak)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;56 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>57&#45;&gt;56</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1534,-434.9735C1534,-423.1918 1534,-407.5607 1534,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1537.5001,-394.0033 1534,-384.0034 1530.5001,-394.0034 1537.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1562.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 58 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>58</title>\n",
       "<text text-anchor=\"middle\" x=\"1641\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">58 (untuk)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;58 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>57&#45;&gt;58</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1568.7743,-434.9349C1577.6835,-429.6579 1587.0008,-423.5354 1595,-417 1603.9567,-409.6823 1612.7301,-400.5879 1620.1732,-392.114\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1623.0449,-394.1402 1626.8644,-384.2577 1617.7158,-389.6014 1623.0449,-394.1402\"/>\n",
       "<text text-anchor=\"middle\" x=\"1627\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 52 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>52</title>\n",
       "<text text-anchor=\"middle\" x=\"1141\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">52 (Sri)</text>\n",
       "</g>\n",
       "<!-- 52&#45;&gt;52 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>52&#45;&gt;52</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1176.7532,-808.8361C1186.7932,-808.4929 1194.5,-805.8809 1194.5,-801 1194.5,-797.9495 1191.4895,-795.7852 1186.7805,-794.5072\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1187.1294,-791.0227 1176.7532,-793.1639 1186.1999,-797.9608 1187.1294,-791.0227\"/>\n",
       "<text text-anchor=\"middle\" x=\"1206.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 54 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>54</title>\n",
       "<text text-anchor=\"middle\" x=\"1141\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">54 (Yassin)</text>\n",
       "</g>\n",
       "<!-- 52&#45;&gt;54 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>52&#45;&gt;54</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1141,-782.9735C1141,-771.1918 1141,-755.5607 1141,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1144.5001,-742.0033 1141,-732.0034 1137.5001,-742.0034 1144.5001,-742.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1153\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 59 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>59</title>\n",
       "<text text-anchor=\"middle\" x=\"1550\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">59 (mengelirukan)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;59 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>59&#45;&gt;59</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1624.3601,-808.067C1634.9164,-806.924 1642,-804.5684 1642,-801 1642,-798.714 1639.0929,-796.9257 1634.228,-795.6351\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1634.8096,-792.1838 1624.3601,-793.933 1633.6196,-799.082 1634.8096,-792.1838\"/>\n",
       "<text text-anchor=\"middle\" x=\"1652.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 60 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>60</title>\n",
       "<text text-anchor=\"middle\" x=\"1550\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">60 (rakyat)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;60 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>59&#45;&gt;60</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1550,-782.9735C1550,-771.1918 1550,-755.5607 1550,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1553.5001,-742.0033 1550,-732.0034 1546.5001,-742.0034 1553.5001,-742.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1561\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 62 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>62</title>\n",
       "<text text-anchor=\"middle\" x=\"1848\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">62 (Wan)</text>\n",
       "</g>\n",
       "<!-- 63 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>63</title>\n",
       "<text text-anchor=\"middle\" x=\"1837\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">63 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 62&#45;&gt;63 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>62&#45;&gt;63</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1845.7208,-695.9735C1844.2311,-684.1918 1842.2548,-668.5607 1840.5602,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1844.0031,-654.4853 1839.2763,-645.0034 1837.0584,-655.3634 1844.0031,-654.4853\"/>\n",
       "<text text-anchor=\"middle\" x=\"1856\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 65 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>65</title>\n",
       "<text text-anchor=\"middle\" x=\"1954\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">65 (,)</text>\n",
       "</g>\n",
       "<!-- 66 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>66</title>\n",
       "<text text-anchor=\"middle\" x=\"2046\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">66 (beliau)</text>\n",
       "</g>\n",
       "<!-- 67 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>67</title>\n",
       "<text text-anchor=\"middle\" x=\"2046\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">67 (sudah)</text>\n",
       "</g>\n",
       "<!-- 67&#45;&gt;62 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>67&#45;&gt;62</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1999.7708,-786.5607C1981.8047,-780.5051 1961.2058,-773.0096 1943,-765 1924.3502,-756.795 1904.3859,-746.3478 1887.5962,-737.0482\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1889.0586,-733.8557 1878.6238,-732.0214 1885.6372,-739.9626 1889.0586,-733.8557\"/>\n",
       "<text text-anchor=\"middle\" x=\"1963\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 67&#45;&gt;65 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>67&#45;&gt;65</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2022.0819,-782.966C2015.1025,-777.4133 2007.5856,-771.1408 2001,-765 1992.5915,-757.1593 1983.9169,-748.0725 1976.3445,-739.76\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1978.7498,-737.1998 1969.4668,-732.0934 1973.5393,-741.8743 1978.7498,-737.1998\"/>\n",
       "<text text-anchor=\"middle\" x=\"2021.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 67&#45;&gt;66 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>67&#45;&gt;66</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2046,-782.9735C2046,-771.1918 2046,-755.5607 2046,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2049.5001,-742.0033 2046,-732.0034 2042.5001,-742.0034 2049.5001,-742.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2066\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 67&#45;&gt;67 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>67&#45;&gt;67</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2092.0242,-808.6827C2102.4063,-807.9906 2110,-805.4297 2110,-801 2110,-798.2314 2107.0337,-796.1929 2102.2743,-794.8843\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2102.4383,-791.3688 2092.0242,-793.3173 2101.3804,-798.2884 2102.4383,-791.3688\"/>\n",
       "<text text-anchor=\"middle\" x=\"2138.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 68 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>68</title>\n",
       "<text text-anchor=\"middle\" x=\"2175\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">68 (menjangka)</text>\n",
       "</g>\n",
       "<!-- 67&#45;&gt;68 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>67&#45;&gt;68</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2072.7289,-782.9735C2092.1875,-769.8503 2118.7275,-751.9512 2139.8948,-737.6756\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2141.9716,-740.4966 2148.3054,-732.0034 2138.0576,-734.6931 2141.9716,-740.4966\"/>\n",
       "<text text-anchor=\"middle\" x=\"2138\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 69 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>69</title>\n",
       "<text text-anchor=\"middle\" x=\"2295\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">69 (ada)</text>\n",
       "</g>\n",
       "<!-- 67&#45;&gt;69 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>67&#45;&gt;69</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2092.091,-787.2538C2113.1901,-780.7866 2138.4603,-772.7951 2161,-765 2197.3355,-752.4338 2207.9054,-747.4405 2247.7319,-732.115\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2249.2073,-735.2979 2257.294,-728.4528 2246.7037,-728.761 2249.2073,-735.2979\"/>\n",
       "<text text-anchor=\"middle\" x=\"2225.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 70 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>70</title>\n",
       "<text text-anchor=\"middle\" x=\"1962\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">70 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;70 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>68&#45;&gt;70</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2130.8663,-695.9735C2097.3089,-682.267 2050.9966,-663.3507 2015.3587,-648.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2016.6582,-645.5445 2006.0772,-645.0034 2014.0113,-652.0248 2016.6582,-645.5445\"/>\n",
       "<text text-anchor=\"middle\" x=\"2093\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 77 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>77</title>\n",
       "<text text-anchor=\"middle\" x=\"2183\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">77 (berhubung)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;77 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>68&#45;&gt;77</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2176.6576,-695.9735C2177.741,-684.1918 2179.1783,-668.5607 2180.4108,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2183.914,-655.2819 2181.3445,-645.0034 2176.9435,-654.6408 2183.914,-655.2819\"/>\n",
       "<text text-anchor=\"middle\" x=\"2190.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 83 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>83</title>\n",
       "<text text-anchor=\"middle\" x=\"2292\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">83 (.)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;83 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>68&#45;&gt;83</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2199.2425,-695.9735C2216.7341,-682.9669 2240.5348,-665.269 2259.6473,-651.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2261.8524,-653.7791 2267.7886,-645.0034 2257.6755,-648.1618 2261.8524,-653.7791\"/>\n",
       "<text text-anchor=\"middle\" x=\"2261.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 71 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>71</title>\n",
       "<text text-anchor=\"middle\" x=\"1745\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">71 (balas)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;71 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>70&#45;&gt;71</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1905.2185,-608.9207C1889.1809,-603.4895 1871.7986,-597.2927 1856,-591 1834.3141,-582.3623 1810.7565,-571.7244 1790.9456,-562.403\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1792.373,-559.2063 1781.8375,-558.0865 1789.3751,-565.5319 1792.373,-559.2063\"/>\n",
       "<text text-anchor=\"middle\" x=\"1876\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 73 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>73</title>\n",
       "<text text-anchor=\"middle\" x=\"1851\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">73 (Najib)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;73 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>70&#45;&gt;73</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1939.0007,-608.9735C1922.4805,-596.0253 1900.0284,-578.4276 1881.9378,-564.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1883.9995,-561.4175 1873.9698,-558.0034 1879.6813,-566.9269 1883.9995,-561.4175\"/>\n",
       "<text text-anchor=\"middle\" x=\"1925\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 75 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>75</title>\n",
       "<text text-anchor=\"middle\" x=\"1962\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">75 (tulisan)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;75 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>70&#45;&gt;75</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1962,-608.9735C1962,-597.1918 1962,-581.5607 1962,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1965.5001,-568.0033 1962,-558.0034 1958.5001,-568.0034 1965.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1973\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 78 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>78</title>\n",
       "<text text-anchor=\"middle\" x=\"2075\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">78 (kesan)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;78 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>77&#45;&gt;78</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2160.6223,-608.9735C2144.6934,-596.1419 2123.0964,-578.7443 2105.5784,-564.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2107.3322,-561.551 2097.349,-558.0034 2102.9409,-567.0023 2107.3322,-561.551\"/>\n",
       "<text text-anchor=\"middle\" x=\"2147\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 79 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>79</title>\n",
       "<text text-anchor=\"middle\" x=\"2186\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">79 (positif)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;79 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>77&#45;&gt;79</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2183.6216,-608.9735C2184.0279,-597.1918 2184.5669,-581.5607 2185.029,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2188.5324,-568.1181 2185.3792,-558.0034 2181.5366,-567.8768 2188.5324,-568.1181\"/>\n",
       "<text text-anchor=\"middle\" x=\"2205\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 80 -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>80</title>\n",
       "<text text-anchor=\"middle\" x=\"2304\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">80 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;80 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>77&#45;&gt;80</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2208.0713,-608.9735C2226.242,-595.9086 2250.9962,-578.1102 2270.8068,-563.8662\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2272.8849,-566.6829 2278.9608,-558.0034 2268.7984,-560.9994 2272.8849,-566.6829\"/>\n",
       "<text text-anchor=\"middle\" x=\"2288\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 81 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>81</title>\n",
       "<text text-anchor=\"middle\" x=\"2440\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">81 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;81 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>77&#45;&gt;81</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2247.4204,-611.1199C2286.5129,-601.4826 2328.9336,-591.0232 2329,-591 2351.6407,-583.0872 2375.9396,-572.2129 2396.0134,-562.5377\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2397.7589,-565.5804 2405.2094,-558.0477 2394.6876,-559.2902 2397.7589,-565.5804\"/>\n",
       "<text text-anchor=\"middle\" x=\"2404\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 72 -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>72</title>\n",
       "<text text-anchor=\"middle\" x=\"1828\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">72 (daripada)</text>\n",
       "</g>\n",
       "<!-- 73&#45;&gt;72 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>73&#45;&gt;72</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1846.2344,-521.9735C1843.0888,-510.0751 1838.9054,-494.2508 1835.339,-480.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1838.6992,-479.7766 1832.7595,-471.0034 1831.9317,-481.5658 1838.6992,-479.7766\"/>\n",
       "<text text-anchor=\"middle\" x=\"1857\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 74 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>74</title>\n",
       "<text text-anchor=\"middle\" x=\"1962\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">74 (mengenai)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;74 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>75&#45;&gt;74</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1962,-521.9735C1962,-510.1918 1962,-494.5607 1962,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1965.5001,-481.0033 1962,-471.0034 1958.5001,-481.0034 1965.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1978\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 76 -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>76</title>\n",
       "<text text-anchor=\"middle\" x=\"2086\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">76 (beliau)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;76 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>75&#45;&gt;76</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1987.6929,-521.9735C2006.3142,-508.9086 2031.682,-491.1102 2051.9839,-476.8662\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2054.1641,-479.612 2060.34,-471.0034 2050.1437,-473.8817 2054.1641,-479.612\"/>\n",
       "<text text-anchor=\"middle\" x=\"2052\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 82 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>82</title>\n",
       "<text text-anchor=\"middle\" x=\"2440\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">82 (penuh)</text>\n",
       "</g>\n",
       "<!-- 81&#45;&gt;82 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>81&#45;&gt;82</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2440,-521.9735C2440,-510.1918 2440,-494.5607 2440,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2443.5001,-481.0033 2440,-471.0034 2436.5001,-481.0034 2443.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2460\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f29091327f0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'KUALA LUMPUR: Ketua Penerangan BERSATU, Datuk Wan Saiful Wan Jan membidas kenyataan Datuk Seri Najib Razak dan Ketua Pemuda UMNO, Datuk Dr Asyraf Wajdi Dusuki yang mempertikaikan tindakan kerajaan melaksanakan sekatan pergerakan penuh. Beliau berkata, Najib dan Asyraf Wajdi sengaja memetik kenyataan Perdana Menteri, Tan Sri Muhyiddin Yassin yang tidak lengkap untuk mengelirukan rakyat. Wan Saiful berkata, beliau sudah menjangka ada kenyataan balas daripada Najib mengenai tulisan beliau berhubung kesan positif sekatan pergerakan penuh.'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, mask = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(bert_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(bert_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'albert-tiny-dependency/model.ckpt'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'albert-tiny-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:++++++ warmup starts at step 0, for 29336 steps ++++++\n",
      "INFO:tensorflow:using adamw\n",
      "INFO:tensorflow:Restoring parameters from albert-tiny-dependency/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word, training = False)\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'albert-tiny-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(heads_pred, types_pred, heads, types, lengths,\n",
    "             symbolic_root=False, symbolic_end=False):\n",
    "    batch_size, _ = heads_pred.shape\n",
    "    ucorr = 0.\n",
    "    lcorr = 0.\n",
    "    total = 0.\n",
    "    ucomplete_match = 0.\n",
    "    lcomplete_match = 0.\n",
    "\n",
    "    corr_root = 0.\n",
    "    total_root = 0.\n",
    "    start = 1 if symbolic_root else 0\n",
    "    end = 1 if symbolic_end else 0\n",
    "    for i in range(batch_size):\n",
    "        ucm = 1.\n",
    "        lcm = 1.\n",
    "        for j in range(start, lengths[i] - end):\n",
    "\n",
    "            total += 1\n",
    "            if heads[i, j] == heads_pred[i, j]:\n",
    "                ucorr += 1\n",
    "                if types[i, j] == types_pred[i, j]:\n",
    "                    lcorr += 1\n",
    "                else:\n",
    "                    lcm = 0\n",
    "            else:\n",
    "                ucm = 0\n",
    "                lcm = 0\n",
    "\n",
    "            if heads[i, j] == 0:\n",
    "                total_root += 1\n",
    "                corr_root += 1 if heads_pred[i, j] == 0 else 0\n",
    "\n",
    "        ucomplete_match += ucm\n",
    "        lcomplete_match += lcm\n",
    "    \n",
    "    return ucorr / total, lcorr / total, corr_root / total_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 313/313 [00:37<00:00,  8.27it/s]\n"
     ]
    }
   ],
   "source": [
    "arcs, types, roots = [], [], []\n",
    "real_Y, predict_Y = [], []\n",
    "\n",
    "for i in tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i: index]\n",
    "    batch_x = pad_sequences(batch_x,padding='post')\n",
    "    batch_y = test_Y[i: index]\n",
    "    batch_y = pad_sequences(batch_y,padding='post')\n",
    "    batch_depends = test_depends[i: index]\n",
    "    batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "    \n",
    "    tags_seq, heads = sess.run(\n",
    "        [model.logits, model.heads_seq],\n",
    "        feed_dict = {\n",
    "            model.words: batch_x,\n",
    "        },\n",
    "    )\n",
    "    \n",
    "    arc_accuracy, type_accuracy, root_accuracy = evaluate(heads - 1, tags_seq, batch_depends - 1, batch_y, \n",
    "            np.count_nonzero(batch_x, axis = 1))\n",
    "    arcs.append(arc_accuracy)\n",
    "    types.append(type_accuracy)\n",
    "    roots.append(root_accuracy)\n",
    "    predicted = pred2label(tags_seq)\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_real_Y = []\n",
    "for r in real_Y:\n",
    "    temp_real_Y.extend(r)\n",
    "    \n",
    "temp_predict_Y = []\n",
    "for r in predict_Y:\n",
    "    temp_predict_Y.extend(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "          PAD    1.00000   1.00000   1.00000    627830\n",
      "            X    1.00000   1.00000   1.00000     60741\n",
      "          acl    0.80688   0.76441   0.78507      3192\n",
      "        advcl    0.60052   0.58044   0.59031      1585\n",
      "       advmod    0.92613   0.92570   0.92591      6460\n",
      "         amod    0.86782   0.85927   0.86353      4363\n",
      "        appos    0.76876   0.70271   0.73425      3061\n",
      "         case    0.97508   0.97247   0.97377     10862\n",
      "           cc    0.98150   0.96228   0.97179      3473\n",
      "        ccomp    0.52577   0.29651   0.37918       344\n",
      "     compound    0.85386   0.91240   0.88216     11027\n",
      "compound:plur    0.47826   0.37931   0.42308        29\n",
      "         conj    0.87337   0.85133   0.86221      5112\n",
      "          cop    0.98953   0.95777   0.97339       592\n",
      "        csubj    0.00000   0.00000   0.00000         8\n",
      "          dep    0.59312   0.58640   0.58974       353\n",
      "          det    0.93306   0.90026   0.91637      3840\n",
      "        fixed    0.91270   0.66860   0.77181       172\n",
      "         flat    0.93104   0.95138   0.94110     18492\n",
      "         iobj    0.00000   0.00000   0.00000         1\n",
      "         mark    0.89771   0.92249   0.90993      1703\n",
      "         nmod    0.81300   0.79987   0.80638      4457\n",
      "        nsubj    0.80429   0.82122   0.81267      6891\n",
      "   nsubj:pass    0.77960   0.69890   0.73705      1903\n",
      "       nummod    0.93708   0.93517   0.93612      4427\n",
      "          obj    0.87808   0.88969   0.88385      6128\n",
      "          obl    0.84346   0.79980   0.82105      4965\n",
      "    parataxis    0.43750   0.21448   0.28785       359\n",
      "        punct    0.99444   0.99635   0.99540     20300\n",
      "         root    0.86422   0.89430   0.87901     10000\n",
      "        xcomp    0.67055   0.68068   0.67558      1522\n",
      "\n",
      "     accuracy                        0.98296    824192\n",
      "    macro avg    0.77217   0.73949   0.75253    824192\n",
      " weighted avg    0.98272   0.98296   0.98277    824192\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(temp_real_Y, temp_predict_Y, digits = 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arc accuracy: 0.7865049894214071\n",
      "types accuracy: 0.7587000482179278\n",
      "root accuracy: 1.0\n"
     ]
    }
   ],
   "source": [
    "print('arc accuracy:', np.mean(arcs))\n",
    "print('types accuracy:', np.mean(types))\n",
    "print('root accuracy:', np.mean(roots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or '_seq' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'adam' not in n.name\n",
    "        and 'gradients/bert' not in n.name\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from albert-tiny-dependency/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-32-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 40 variables.\n",
      "INFO:tensorflow:Converted 40 variables to const ops.\n",
      "1723 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('albert-tiny-dependency', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-35-e8b29a78c927>:6: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "tf.set_random_seed(0)\n",
    "\n",
    "pb = 'albert-tiny-dependency/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "if 'bert' in pb:\n",
    "    inputs = ['Placeholder']\n",
    "    a = ['dense/BiasAdd']\n",
    "if 'xlnet' in pb:\n",
    "    inputs = ['Placeholder', 'Placeholder_1', 'Placeholder_2']\n",
    "    a = ['transpose_3']\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits', 'heads_seq'] + a, transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
